#!/bin/tcsh -ef
####
#
#############################################################################
#                                                                           #
# Title: Parallel Processing                                                #
#                                                                           #
# (C) 2dx.org, GNU Plublic License.                                         #
#                                                                           #
# Created..........: 20/09/2007                                             #
# Last Modification: 19.05/2015                                             #
# Author...........: 2dx.org                                                #
#                                                                           #
#############################################################################
#
# SORTORDER: 20
#
# MANUAL: This script allows to launch several jobs in parallel.  Choose below, which 2dx_image scripts to include.
#
# CHECK_IMAGES_OPEN: SELECTED
#
# DISPLAY: script_2dx_initialize_files
# DISPLAY: script_2dx_initialize_files2
# DISPLAY: script_2dx_fftrans
# DISPLAY: script_2dx_getDefTilt
# DISPLAY: script_2dx_getLattice
# DISPLAY: script_2dx_refineLattice
# DISPLAY: script_2dx_getSampleTilt
# DISPLAY: script_2dx_getspots1
# DISPLAY: script_2dx_ctfcor
# DISPLAY: script_2dx_getspots
# DISPLAY: script_2dx_unbend1
# DISPLAY: script_2dx_getspots
# DISPLAY: script_2dx_unbend2
# DISPLAY: script_2dx_unbend_movieA1
# DISPLAY: script_2dx_unbend_movieA2
# DISPLAY: script_2dx_unbend_movieB
# DISPLAY: script_2dx_applyCTF
# DISPLAY: script_2dx_generateMAP
# DISPLAY: script_2dx_initialize
# DISPLAY: script_2dx_evaluateLattice
# DISPLAY: script_2dx_driftcorrect
# DISPLAY: script_2dx_cleanup
# DISPLAY: script_2dx_cleanup2
# DISPLAY: ReUnbend_MP
#
#$end_local_vars
#
set bin_2dx = ""
set proc_2dx = ""
set app_2dx_image = ""
#
set ReUnbend_MP = ""
#
set script_2dx_initialize_files = ""
set script_2dx_initialize_files2 = ""
set script_2dx_fftrans = ""
set script_2dx_getDefTilt = ""
set script_2dx_getLattice = ""
set script_2dx_refineLattice = ""
set script_2dx_getSampleTilt = ""
set script_2dx_getspots1 = ""
set script_2dx_ctfcor = ""
set script_2dx_getspots = ""
set script_2dx_unbend1 = ""
set script_2dx_getspots = ""
set script_2dx_unbend2 = ""
set script_2dx_unbend_movieA1 = ""
set script_2dx_unbend_movieA2 = ""
set script_2dx_unbend_movieB = ""
set script_2dx_applyCTF = ""
set script_2dx_max_likelihood = ""
set script_2dx_generateMAP = ""
set script_2dx_initialize = ""
set script_2dx_evaluateLattice = ""
set script_2dx_cleanup = ""
set script_2dx_cleanup2 = ""
set script_2dx_driftcorrect = ""
set script_2dx_driftcorrect2 = ""
#
#$end_vars 
#
#
set scriptname = 2dx_parallel
\rm -f LOGS/${scriptname}.results
#
set ccp4_setup = 'y'
set noEMAN = 'y'
source ${proc_2dx}/initialize 
source ${proc_2dx}/2dx_merge_makedirs 
#
echo "<<@progress: 1>>"
#
# An alternative is to give scriptnames separated by comma in one string to one run of 2dx_image.
#
echo '#\!/bin/csh -ef' > SCRATCH/${scriptname}_tmp_runjob.com 
echo 'set app_2dx_image = $1' >> SCRATCH/${scriptname}_tmp_runjob.com
echo 'set curdir = $2' >> SCRATCH/${scriptname}_tmp_runjob.com
echo 'set dirfile = $3' >> SCRATCH/${scriptname}_tmp_runjob.com
echo '${app_2dx_image} ${curdir} "2dx_initialize"' >> SCRATCH/${scriptname}_tmp_runjob.com
#
if ( ${script_2dx_initialize_files} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_initialize_files"' >> SCRATCH/${scriptname}_tmp_runjob.com
endif
#
if ( ${script_2dx_driftcorrect} == "y" ) then
  echo '${app_2dx_image} ${curdir} "+2dx_driftcorrect"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_cleanup} == "y" ) then
  echo '${app_2dx_image} ${curdir} "+2dx_cleanup"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_initialize_files2} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_initialize_files"' >> SCRATCH/${scriptname}_tmp_runjob.com
endif
#
if ( ${script_2dx_fftrans} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_fftrans"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_getDefTilt} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_getDefTilt"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_getLattice} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_getLattice"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_refineLattice} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_refineLattice"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_getSampleTilt} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_getSampleTilt"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_getspots1} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_getspots1"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_ctfcor} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_ctfcor"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_getspots} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_getspots"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_unbend1} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_unbend1"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_getspots} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_getspots"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_unbend2} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_unbend2"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_unbend_movieA1} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_unbend_movieA1"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_unbend_movieA2} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_unbend_movieA2"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_unbend_movieB} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_unbend_movieB"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_applyCTF} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_applyCTF"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_max_likelihood} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_max_likelihood"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_generateMAP} == "y" ) then
  echo '${app_2dx_image} ${curdir} "2dx_generateMAP"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_evaluateLattice} == "y" ) then
  echo '${app_2dx_image} ${curdir} "+2dx_evaluateLattice"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_driftcorrect2} == "y" ) then
  echo '${app_2dx_image} ${curdir} "+2dx_driftcorrect"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
if ( ${script_2dx_cleanup2} == "y" ) then
  echo '${app_2dx_image} ${curdir} "+2dx_cleanup"'>> SCRATCH/${scriptname}_tmp_runjob.com
endif
echo 'echo ":: Background process finished with ${dirfile} ("`date`")"' >> SCRATCH/${scriptname}_tmp_runjob.com
#
chmod +x SCRATCH/${scriptname}_tmp_runjob.com
echo " "
echo " "
echo " "
echo "Preparing to launch the following job in parallel:"
cat SCRATCH/${scriptname}_tmp_runjob.com
echo " "
echo " "
echo " "
#
set olddir = ${PWD}
cd ..
set rootdir = ${PWD}
#
############################################################################# 
${proc_2dx}/linblock "Processing custom scripts in all selected directories in ${ReUnbend_MP} jobs"
############################################################################# 
#
cd ${olddir} 
#
# set loadaverage = `uptime | cut -d\: -f4 | cut -d\  -f2`
set loadaverage = `ps -axf | sed -n '/${scriptname}/p' | wc -l`
set ReUnbend_MP = `cat 2dx_merge.cfg | grep "set ReUnbend_MP" | cut -d\" -f2`
\cp -f 2dx_merge.cfg SCRATCH/${scriptname}_2dx_merge_cfg.tmp
#
set dirnum = `cat 2dx_merge_dirfile.dat | wc -l`
echo ": Will work on ${dirnum} image directories."
#
\cp -f 2dx_merge_dirfile.dat SCRATCH/${scriptname}_2dx_merge_dirfile_runfile.dat
#
set dircounter = 1
while ( ${dircounter} <= ${dirnum} )
    #
    set prog_num = `echo ${dircounter} ${dirnum} | awk '{ s = 5 + int( 90 * $1 / $2 ) } END { print s }'` 
    echo "<<@progress: ${prog_num}>>"
    #
    cd ${olddir}
    ${bin_2dx}/2dx_getline.exe << eot > TMP.tmp
SCRATCH/${scriptname}_2dx_merge_dirfile_runfile.dat
${dircounter}
eot
    #
    set dirfile = `cat TMP.tmp`
    \rm TMP.tmp
    #
    cd ..
    cd ${dirfile}
    if ( -e 2dx_image.cfg ) then
      echo ": "
      set curdir = ${PWD}
      # set QVALMA_local = `cat 2dx_image.cfg | grep "set QVALMA =" | cut -d\" -f2`
      # set QVALMB_local = `cat 2dx_image.cfg | grep "set QVALMB =" | cut -d\" -f2`
      # echo ":: QVALMA_local = ${QVALMA_local}" 
      # if ( ${QVALMA_local}x == "x"  || ${QVALMA_local}x == ".x" ||  ${QVALMA_local}x == "0.00x" ||  ${QVALMB_local}x == "x"  || ${QVALMB_local}x == ".x" ||  ${QVALMB_local}x == "0.00x" ) then
      if ( 1 == 1 ) then 
        echo "Working on ${curdir} ("`date`")"

        ${proc_2dx}/linblock "At load ${loadaverage}/${ReUnbend_MP}, launching job ${dircounter} of ${dirnum} on ${dirfile}"

        ###########################################################################################################################
        nohup ${olddir}/SCRATCH/${scriptname}_tmp_runjob.com ${app_2dx_image} ${curdir} ${dirfile} &
        ###########################################################################################################################

        cd ${olddir}
        #
        # Following tells GUI that the parameters for image have been changed and the GUI should reload them.
        echo "<RESETDIR="\"${curdir}\"">" >> LOGS/${scriptname}.results
        #
        set load_too_high = 1
        while ( ${load_too_high} == 1 ) 
          set ReUnbend_MP = `cat SCRATCH/${scriptname}_2dx_merge_cfg.tmp | grep "set ReUnbend_MP" | cut -d\" -f2`
          set loadaverage = `ps -axf | sed -n '/runjob/p' | wc -l`
          set pythonJobs = `ps -axf | sed -n '/python/p' | wc -l`
          if ( `uname` == "Linux" ) then
            set memfree = `vmstat 1 1 | awk 'NR>2 {gsub("K","000");print ($1+$4)/256000}'`
          else
            set memfree = `vm_stat -c 1 1 | awk 'NR>2 {gsub("K","000");print ($1+$4)/256000}'`
          endif
          set load_too_high = `echo ${loadaverage} ${ReUnbend_MP} | awk '{ if ( ( $1 - 0 ) > $2 ) { s = 1 } else { s = 0 } } END { print s }'`
          set python_too_high = `echo ${pythonJobs} ${ReUnbend_MP} | awk '{ if ( ( $1 - 0 ) > $2 ) { s = 1 } else { s = 0 } } END { print s }'`
          set mem_too_low = `echo ${memfree} | awk '{ if ( $1 < 20.0 ) { s = 1 } else { s = 0 } } END { print s }'`
          if ( ${load_too_high} == 1 || ${mem_too_low} == 1 || ${python_too_high} == 1 ) then
            sleep 2
          endif
        end

      else
        echo ": Skipping ${dirfile}: QVALMA = ${QVALMA_local},  QVALMB = ${QVALMB_local} ("`date`")"
      endif
    else
      echo "::ERROR for ${dirfile}: No 2dx_image.cfg found."
    endif
    # 
    # 
    @ dircounter++
    cd ${olddir}
end
echo "::Waiting for last jobs to finish"
set load_too_high = 1
while ( ${load_too_high} == 1 ) 
    set ReUnbend_MP = `cat SCRATCH/${scriptname}_2dx_merge_cfg.tmp | grep "set ReUnbend_MP" | cut -d\" -f2`
    set loadaverage = `ps -axf | sed -n '/runjob/p' | wc -l`
    set pythonJobs = `ps -axf | sed -n '/python/p' | wc -l`
    set load_too_high = `echo ${loadaverage} 1 | awk '{ if ( ( $1 - 2 ) > $2 ) { s = 1 } else { s = 0 } } END { print s }'`
    set python_too_high = `echo ${pythonJobs} ${ReUnbend_MP} | awk '{ if ( ( $1 - 0 ) > $2 ) { s = 1 } else { s = 0 } } END { print s }'`
    if ( ${load_too_high} == 1 || ${python_too_high} == 1 ) then
      sleep 5
      # ps -af | grep ${scriptname}
    endif
end
echo "::Last jobs have finished."
#
#
echo ":: Done."
echo "<<@progress: 100>>"
#
############################################################################# 
${proc_2dx}/linblock "${scriptname} - normal end."
#############################################################################
#
exit
#
#
